{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "heading_collapsed": true
   },
   "source": [
    "# Start - Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Step1 - Get repo\n",
    "! rm -rf CS4180-DL\n",
    "! git clone --branch feature/prerak2 https://github.com/prerakmody/CS4180-DL\n",
    "\n",
    "# Step2 - Get Dataset\n",
    "! wget -P /content/CS4180-DL/data/dataset https://pjreddie.com/media/files/VOCtrainval_11-May-2012.tar\n",
    "! tar xf /content/CS4180-DL/data/dataset/VOCtrainval_11-May-2012.tar --directory /content/CS4180-DL/data/dataset\n",
    "! wget -P /content/CS4180-DL/data/dataset https://pjreddie.com/media/files/VOCtrainval_06-Nov-2007.tar\n",
    "! tar xf /content/CS4180-DL/data/dataset/VOCtrainval_06-Nov-2007.tar --directory /content/CS4180-DL/data/dataset\n",
    "! wget -P /content/CS4180-DL/data/dataset https://pjreddie.com/media/files/VOCtest_06-Nov-2007.tar\n",
    "! tar xf /content/CS4180-DL/data/dataset/VOCtest_06-Nov-2007.tar --directory /content/CS4180-DL/data/dataset\n",
    "! mkdir /content/CS4180-DL/data/weights\n",
    "\n",
    "! wget -P /content/CS4180-DL/data/weights https://pjreddie.com/media/files/yolov2-voc.weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T09:28:23.399092Z",
     "start_time": "2019-05-28T09:28:18.026766Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n",
      " - year :  2012  || image_set :  train\n",
      " - year :  2012  || image_set :  val\n",
      " - year :  2007  || image_set :  train\n",
      " - year :  2007  || image_set :  val\n",
      " - year :  2007  || image_set :  test\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "## Step3 - Add path to main dir\n",
    "DIR_MAIN = os.path.abspath('CS4180-DL/') #'../../../CS4180-DL/'\n",
    "sys.path.append(DIR_MAIN)\n",
    "\n",
    "## Step4 - Download pruned weights from GDrive\n",
    "from src.utils import download_gdrive\n",
    "FILE_ID =  '1dHnUQ8G3GObZSMh9eQ0zUR5wor0ttW3U'\n",
    "DIR_WEIGHTS = os.path.join(DIR_MAIN, 'data/weights/pruned')\n",
    "! mkdir {DIR_WEIGHTS}\n",
    "DEST_NAME_ZIP   = os.path.join(DIR_WEIGHTS, 'weights-prune-me.zip')\n",
    "DEST_NAME_FILES = DIR_WEIGHTS\n",
    "download_gdrive(FILE_ID, DEST_NAME_ZIP)\n",
    "! unzip {DEST_NAME_ZIP} -d {DEST_NAME_FILES}\n",
    "\n",
    "## Step5 - Generate .txt files for training/validation \n",
    "from src.dataloader import setup_VOC\n",
    "DIR_DATA = os.path.join(DIR_MAIN, 'data/dataset/')\n",
    "setup_VOC(DIR_DATA)\n",
    "DIR_DATA_VOC = os.path.join(DIR_DATA, 'VOCdevkit')\n",
    "! cat {DIR_DATA_VOC}/2007_train.txt {DIR_DATA_VOC}/2007_val.txt {DIR_DATA_VOC}/2012_*.txt > {DIR_DATA_VOC}/voc_train.txt\n",
    "\n",
    "## Step6 - Check which GPU\n",
    "! nvidia-smi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Start - Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "\n",
    "import torch\n",
    "\n",
    "dir_main = os.path.abspath('CS4180-DL')\n",
    "sys.path.append(dir_main)\n",
    "print (' - In Path : ', sys.path[-1])\n",
    "\n",
    "USE_GPU = torch.cuda.is_available()\n",
    "print (' - USE_GPU : ', USE_GPU)\n",
    "\n",
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Start - Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "torch.cuda.empty_cache()\n",
    "from src.predict import *\n",
    "from src.nets import *\n",
    "\n",
    "if (1):\n",
    "    DIR_PROJ         = 'CS4180-DL'\n",
    "\n",
    "if (1):\n",
    "    MODEL            = ''\n",
    "    MODEL_CFGFILE    = os.path.join(DIR_PROJ, 'data/cfg/github_pjreddie/yolov2-voc.cfg')\n",
    "    \n",
    "    MODEL_WEIGHTFILE = os.path.join(DIR_PROJ, 'data/weights/yolov2-voc.weights') #['0.5' : 0.6366, '0.25' : 0.7, '0.1' : 0.7363]\n",
    "    EVAL_PREFIX      = 'iter1_pretrained_'\n",
    "    \n",
    "    # MODEL_WEIGHTFILE = os.path.join(DIR_PROJ, 'data/weights/pruned/yolov2-voc-weight-prune-10.0.weights') #['0.5' : 0.6323, '0.25' : 0.7]\n",
    "    # EVAL_PREFIX      = 'iter1_pretrained_10_'\n",
    "    \n",
    "    # MODEL_WEIGHTFILE = os.path.join(DIR_PROJ, 'data/weights/pruned/yolov2-voc-weight-prune-30.0.weights') #['0.5' : 0.6323, '0.25' : 0.69, ]\n",
    "    # EVAL_PREFIX      = 'iter1_pretrained_30_'\n",
    "    \n",
    "    # MODEL_WEIGHTFILE = os.path.join(DIR_PROJ, 'data/weights/pruned/yolov2-voc-weight-prune-50.0.weights') #['0.25' :  0.6874]\n",
    "    # EVAL_PREFIX      = 'iter1_pretrained_50_'\n",
    "    \n",
    "    # MODEL_WEIGHTFILE = os.path.join(DIR_PROJ, 'data/weights/pruned/yolov2-voc-weight-prune-70.0.weights') #['0.25' :  0.5160]\n",
    "    # EVAL_PREFIX      = 'iter1_pretrained_70_'\n",
    "    \n",
    "    # MODEL_WEIGHTFILE = os.path.join(DIR_PROJ, 'data/weights/pruned/yolov2-voc-weight-prune-90.0.weights') #['0.25' :  0, '0.1' : 0, '0.005' : 0]\n",
    "    # EVAL_PREFIX      = 'iter1_pretrained_90_'\n",
    "    \n",
    "    MODEL_LOSS       = RegionLoss()\n",
    "    \n",
    "    PASCAL_DIR       = os.path.join(DIR_PROJ, 'data/dataset/VOCdevkit/')\n",
    "    EVAL_IMAGELIST   = os.path.join(DIR_PROJ, 'data/dataset/VOCdevkit/2007_test.txt')\n",
    "    EVAL_OUTPUTDIR   = os.path.join(DIR_PROJ, 'eval_data')\n",
    "    EVAL_OUTPUTDIR_PKL = os.path.join(DIR_PROJ, 'eval_results')\n",
    "    \n",
    "    \n",
    "    # EVAL_PREFIX      = 'iter1_weighspruned_70_'\n",
    "    \n",
    "    print (' - 0. MODEL       : ', MODEL)\n",
    "    print (' - 0. MODEL_WEIGHT: ', MODEL_WEIGHTFILE)\n",
    "    print (' - 0. EVAL_PREFIX : ', EVAL_PREFIX)\n",
    "   \n",
    "if (1):\n",
    "    BATCH_SIZE = 32\n",
    "    CONF_THRESH = 0.05 # [0.5, 0.25, 0.1, 0.005]\n",
    "    print (' - 0. BATCH_SIZE  : ', BATCH_SIZE)\n",
    "    print (' - 0. CONF_THRESH : ', CONF_THRESH)\n",
    "    \n",
    "if (1):\n",
    "    try:\n",
    "        print (' - 0. Logger      : ', LOGGER)\n",
    "    except:\n",
    "        LOGGER = TensorBoardColab()\n",
    "        print (' - 0. Logger      : ', LOGGER)    \n",
    "    \n",
    "    print ('')\n",
    "\n",
    "\n",
    "valObj = PASCALVOCEval(MODEL, MODEL_CFGFILE, MODEL_WEIGHTFILE, MODEL_LOSS \n",
    "                       ,PASCAL_DIR, EVAL_IMAGELIST, EVAL_OUTPUTDIR, EVAL_PREFIX, EVAL_OUTPUTDIR_PKL\n",
    "                       , LOGGER)\n",
    "valObj.predict(BATCH_SIZE=BATCH_SIZE, CONF_THRESH=CONF_THRESH)\n",
    "# valObj._do_python_eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Start - Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-01T12:20:45.545245Z",
     "start_time": "2019-06-01T12:20:45.521169Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "UsageError: Line magic function `%` not found.\n"
     ]
    }
   ],
   "source": [
    "% reset\n",
    "import os\n",
    "import sys\n",
    "\n",
    "import torch\n",
    "from tensorboardcolab import TensorBoardColab\n",
    "\n",
    "torch.cuda.empty_cache()\n",
    "\n",
    "\n",
    "\n",
    "if (torch.cuda.is_available()):\n",
    "    \n",
    "    if (1):\n",
    "        DIR_MAIN         = os.path.abspath('CS4180-DL')\n",
    "        sys.path.append(DIR_MAIN)\n",
    "        print (' - 0. DIR_MAIN     :  ', DIR_MAIN)\n",
    "        from src.train import YOLOv2Train\n",
    "        \n",
    "    if (1):\n",
    "        PASCAL_DIR   = os.path.join(DIR_MAIN, 'data/dataset/VOCdevkit/')\n",
    "        PASCAL_TRAIN = os.path.join(DIR_MAIN, 'data/dataset/VOCdevkit/voc_train.txt')\n",
    "        PASCAL_VALID = os.path.join(DIR_MAIN, 'data/dataset/VOCdevkit/2007_test.txt')\n",
    "        TRAIN_LOGDIR = os.path.join(DIR_MAIN, 'train_data')\n",
    "        VAL_LOGDIR   = os.path.join(DIR_MAIN, 'eval_data')\n",
    "        VAL_OUTPUTDIR_PKL = os.path.join(DIR_MAIN, 'eval_results')\n",
    "        MODEL_CFG    = os.path.join(DIR_MAIN, 'data/cfg/github_pjreddie/yolov2-voc.cfg')\n",
    "        \n",
    "    if (1):\n",
    "        MODEL_WEIGHT = os.path.join(DIR_MAIN, 'data/weights/pruned/yolov2-voc-weight-prune-30.0.weights')\n",
    "        VAL_PREFIX   = 'pretrained-30_'\n",
    "        \n",
    "        # MODEL_WEIGHT = os.path.join(DIR_MAIN, 'data/weights/yolov2-voc.weights')\n",
    "        # VAL_PREFIX   = 'pretrained_'\n",
    "        \n",
    "        print (' - 0. MODEL_WEIGHT :  ', MODEL_WEIGHT)\n",
    "        print (' - 0. VAL_PREFIX   : ', VAL_PREFIX)\n",
    "        \n",
    "    if (1):\n",
    "        BATCH_SIZE   = 32;\n",
    "        print (' - 0. BATCH_SIZE   : ', BATCH_SIZE)\n",
    "\n",
    "    if (1):\n",
    "        DEBUG_EPOCHS = 50 #[-1, 50]\n",
    "        print (' - 0. DEBUG_EPOCHS : ', DEBUG_EPOCHS)\n",
    "        try:\n",
    "            print (' - 0. Logger       : ', LOGGER)\n",
    "        except:\n",
    "            LOGGER = TensorBoardColab()\n",
    "            print (' - 0. Logger       : ', LOGGER)\n",
    "        print ('')\n",
    "\n",
    "\n",
    "    if (1):\n",
    "        trainObj = YOLOv2Train()\n",
    "        trainObj.train(PASCAL_DIR, PASCAL_TRAIN, PASCAL_VALID, TRAIN_LOGDIR, VAL_LOGDIR, VAL_OUTPUTDIR_PKL, VAL_PREFIX\n",
    "                       , MODEL_CFG, MODEL_WEIGHT\n",
    "                       , BATCH_SIZE\n",
    "                       , LOGGER, DEBUG_EPOCHS=DEBUG_EPOCHS, verbose=0)\n",
    "else:\n",
    "    print (' - GPU Issues!!')"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
